Chapter 11 - Fine-Tuning BERT

Data

from datasets import load_dataset

# Prepare data and splits
tomatoes = load_dataset("rotten_tomatoes")
train_data, test_data = tomatoes["train"], tomatoes["test"]

Supervised Classification

HuggingFace Trainer

from transformers import AutoTokenizer, AutoModelForSequenceClassification

# Load Model and Tokenizer
model_id = "bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Tokenize our data.

from transformers import DataCollatorWithPadding

# Pad to the longest sequence in the batch
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

def preprocess_function(examples):
   """Tokenize input data"""
   return tokenizer(examples["text"], truncation=True)

# Tokenize train/test data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_test = test_data.map(preprocess_function, batched=True)

Define metrics.

import numpy as np
import evaluate


def compute_metrics(eval_pred):
    """Calculate F1 score"""
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)

    load_f1 = evaluate.load("f1")
    f1 = load_f1.compute(predictions=predictions, references=labels)["f1"]
    return {"f1": f1}

Train model.

from transformers import TrainingArguments, Trainer

# Training arguments for parameter tuning
training_args = TrainingArguments(
   "model",
   learning_rate=2e-5,
   per_device_train_batch_size=16,
   per_device_eval_batch_size=16,
   num_train_epochs=1,
   weight_decay=0.01,
   save_strategy="epoch",
   report_to="none"
)

# Trainer which executes the training process
trainer = Trainer(
   model=model,
   args=training_args,
   train_dataset=tokenized_train,
   eval_dataset=tokenized_test,
   tokenizer=tokenizer,
   data_collator=data_collator,
   compute_metrics=compute_metrics,
)
trainer.train()
[534/534 01:00, Epoch 1/1]
Step Training Loss
500 0.424000 

TrainOutput(global_step=534, training_loss=0.4183677966228585, metrics={'train_runtime': 61.6658, 'train_samples_per_second': 138.326, 'train_steps_per_second': 8.66, 'total_flos': 227605451772240.0, 'train_loss': 0.4183677966228585, 'epoch': 1.0})

Evaluate results.

trainer.evaluate()
[67/67 00:01] 
{'eval_loss': 0.37090229988098145,
 'eval_f1': 0.8566073102155576,
 'eval_runtime': 3.1133,
 'eval_samples_per_second': 342.407,
 'eval_steps_per_second': 21.521,
 'epoch': 1.0}

Freeze Layers

# Load Model and Tokenizer
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
# Print layer names
for name, param in model.named_parameters():
    print(name)
bert.embeddings.word_embeddings.weight
bert.embeddings.position_embeddings.weight
bert.embeddings.token_type_embeddings.weight
bert.embeddings.LayerNorm.weight
bert.embeddings.LayerNorm.bias
bert.encoder.layer.0.attention.self.query.weight
bert.encoder.layer.0.attention.self.query.bias
bert.encoder.layer.0.attention.self.key.weight
bert.encoder.layer.0.attention.self.key.bias
bert.encoder.layer.0.attention.self.value.weight
bert.encoder.layer.0.attention.self.value.bias
bert.encoder.layer.0.attention.output.dense.weight
bert.encoder.layer.0.attention.output.dense.bias
bert.encoder.layer.0.attention.output.LayerNorm.weight
bert.encoder.layer.0.attention.output.LayerNorm.bias
bert.encoder.layer.0.intermediate.dense.weight
bert.encoder.layer.0.intermediate.dense.bias
bert.encoder.layer.0.output.dense.weight
bert.encoder.layer.0.output.dense.bias
bert.encoder.layer.0.output.LayerNorm.weight
bert.encoder.layer.0.output.LayerNorm.bias
bert.encoder.layer.1.attention.self.query.weight
bert.encoder.layer.1.attention.self.query.bias
bert.encoder.layer.1.attention.self.key.weight
bert.encoder.layer.1.attention.self.key.bias
bert.encoder.layer.1.attention.self.value.weight
bert.encoder.layer.1.attention.self.value.bias
bert.encoder.layer.1.attention.output.dense.weight
bert.encoder.layer.1.attention.output.dense.bias
bert.encoder.layer.1.attention.output.LayerNorm.weight
bert.encoder.layer.1.attention.output.LayerNorm.bias
bert.encoder.layer.1.intermediate.dense.weight
bert.encoder.layer.1.intermediate.dense.bias
bert.encoder.layer.1.output.dense.weight
bert.encoder.layer.1.output.dense.bias
bert.encoder.layer.1.output.LayerNorm.weight
bert.encoder.layer.1.output.LayerNorm.bias
bert.encoder.layer.2.attention.self.query.weight
bert.encoder.layer.2.attention.self.query.bias
bert.encoder.layer.2.attention.self.key.weight
bert.encoder.layer.2.attention.self.key.bias
bert.encoder.layer.2.attention.self.value.weight
bert.encoder.layer.2.attention.self.value.bias
bert.encoder.layer.2.attention.output.dense.weight
bert.encoder.layer.2.attention.output.dense.bias
bert.encoder.layer.2.attention.output.LayerNorm.weight
bert.encoder.layer.2.attention.output.LayerNorm.bias
bert.encoder.layer.2.intermediate.dense.weight
bert.encoder.layer.2.intermediate.dense.bias
bert.encoder.layer.2.output.dense.weight
bert.encoder.layer.2.output.dense.bias
bert.encoder.layer.2.output.LayerNorm.weight
bert.encoder.layer.2.output.LayerNorm.bias
bert.encoder.layer.3.attention.self.query.weight
bert.encoder.layer.3.attention.self.query.bias
bert.encoder.layer.3.attention.self.key.weight
bert.encoder.layer.3.attention.self.key.bias
bert.encoder.layer.3.attention.self.value.weight
bert.encoder.layer.3.attention.self.value.bias
bert.encoder.layer.3.attention.output.dense.weight
bert.encoder.layer.3.attention.output.dense.bias
bert.encoder.layer.3.attention.output.LayerNorm.weight
bert.encoder.layer.3.attention.output.LayerNorm.bias
bert.encoder.layer.3.intermediate.dense.weight
bert.encoder.layer.3.intermediate.dense.bias
bert.encoder.layer.3.output.dense.weight
bert.encoder.layer.3.output.dense.bias
bert.encoder.layer.3.output.LayerNorm.weight
bert.encoder.layer.3.output.LayerNorm.bias
bert.encoder.layer.4.attention.self.query.weight
bert.encoder.layer.4.attention.self.query.bias
bert.encoder.layer.4.attention.self.key.weight
bert.encoder.layer.4.attention.self.key.bias
bert.encoder.layer.4.attention.self.value.weight
bert.encoder.layer.4.attention.self.value.bias
bert.encoder.layer.4.attention.output.dense.weight
bert.encoder.layer.4.attention.output.dense.bias
bert.encoder.layer.4.attention.output.LayerNorm.weight
bert.encoder.layer.4.attention.output.LayerNorm.bias
bert.encoder.layer.4.intermediate.dense.weight
bert.encoder.layer.4.intermediate.dense.bias
bert.encoder.layer.4.output.dense.weight
bert.encoder.layer.4.output.dense.bias
bert.encoder.layer.4.output.LayerNorm.weight
bert.encoder.layer.4.output.LayerNorm.bias
bert.encoder.layer.5.attention.self.query.weight
bert.encoder.layer.5.attention.self.query.bias
bert.encoder.layer.5.attention.self.key.weight
bert.encoder.layer.5.attention.self.key.bias
bert.encoder.layer.5.attention.self.value.weight
bert.encoder.layer.5.attention.self.value.bias
bert.encoder.layer.5.attention.output.dense.weight
bert.encoder.layer.5.attention.output.dense.bias
bert.encoder.layer.5.attention.output.LayerNorm.weight
bert.encoder.layer.5.attention.output.LayerNorm.bias
bert.encoder.layer.5.intermediate.dense.weight
bert.encoder.layer.5.intermediate.dense.bias
bert.encoder.layer.5.output.dense.weight
bert.encoder.layer.5.output.dense.bias
bert.encoder.layer.5.output.LayerNorm.weight
bert.encoder.layer.5.output.LayerNorm.bias
bert.encoder.layer.6.attention.self.query.weight
bert.encoder.layer.6.attention.self.query.bias
bert.encoder.layer.6.attention.self.key.weight
bert.encoder.layer.6.attention.self.key.bias
bert.encoder.layer.6.attention.self.value.weight
bert.encoder.layer.6.attention.self.value.bias
bert.encoder.layer.6.attention.output.dense.weight
bert.encoder.layer.6.attention.output.dense.bias
bert.encoder.layer.6.attention.output.LayerNorm.weight
bert.encoder.layer.6.attention.output.LayerNorm.bias
bert.encoder.layer.6.intermediate.dense.weight
bert.encoder.layer.6.intermediate.dense.bias
bert.encoder.layer.6.output.dense.weight
bert.encoder.layer.6.output.dense.bias
bert.encoder.layer.6.output.LayerNorm.weight
bert.encoder.layer.6.output.LayerNorm.bias
bert.encoder.layer.7.attention.self.query.weight
bert.encoder.layer.7.attention.self.query.bias
bert.encoder.layer.7.attention.self.key.weight
bert.encoder.layer.7.attention.self.key.bias
bert.encoder.layer.7.attention.self.value.weight
bert.encoder.layer.7.attention.self.value.bias
bert.encoder.layer.7.attention.output.dense.weight
bert.encoder.layer.7.attention.output.dense.bias
bert.encoder.layer.7.attention.output.LayerNorm.weight
bert.encoder.layer.7.attention.output.LayerNorm.bias
bert.encoder.layer.7.intermediate.dense.weight
bert.encoder.layer.7.intermediate.dense.bias
bert.encoder.layer.7.output.dense.weight
bert.encoder.layer.7.output.dense.bias
bert.encoder.layer.7.output.LayerNorm.weight
bert.encoder.layer.7.output.LayerNorm.bias
bert.encoder.layer.8.attention.self.query.weight
bert.encoder.layer.8.attention.self.query.bias
bert.encoder.layer.8.attention.self.key.weight
bert.encoder.layer.8.attention.self.key.bias
bert.encoder.layer.8.attention.self.value.weight
bert.encoder.layer.8.attention.self.value.bias
bert.encoder.layer.8.attention.output.dense.weight
bert.encoder.layer.8.attention.output.dense.bias
bert.encoder.layer.8.attention.output.LayerNorm.weight
bert.encoder.layer.8.attention.output.LayerNorm.bias
bert.encoder.layer.8.intermediate.dense.weight
bert.encoder.layer.8.intermediate.dense.bias
bert.encoder.layer.8.output.dense.weight
bert.encoder.layer.8.output.dense.bias
bert.encoder.layer.8.output.LayerNorm.weight
bert.encoder.layer.8.output.LayerNorm.bias
bert.encoder.layer.9.attention.self.query.weight
bert.encoder.layer.9.attention.self.query.bias
bert.encoder.layer.9.attention.self.key.weight
bert.encoder.layer.9.attention.self.key.bias
bert.encoder.layer.9.attention.self.value.weight
bert.encoder.layer.9.attention.self.value.bias
bert.encoder.layer.9.attention.output.dense.weight
bert.encoder.layer.9.attention.output.dense.bias
bert.encoder.layer.9.attention.output.LayerNorm.weight
bert.encoder.layer.9.attention.output.LayerNorm.bias
bert.encoder.layer.9.intermediate.dense.weight
bert.encoder.layer.9.intermediate.dense.bias
bert.encoder.layer.9.output.dense.weight
bert.encoder.layer.9.output.dense.bias
bert.encoder.layer.9.output.LayerNorm.weight
bert.encoder.layer.9.output.LayerNorm.bias
bert.encoder.layer.10.attention.self.query.weight
bert.encoder.layer.10.attention.self.query.bias
bert.encoder.layer.10.attention.self.key.weight
bert.encoder.layer.10.attention.self.key.bias
bert.encoder.layer.10.attention.self.value.weight
bert.encoder.layer.10.attention.self.value.bias
bert.encoder.layer.10.attention.output.dense.weight
bert.encoder.layer.10.attention.output.dense.bias
bert.encoder.layer.10.attention.output.LayerNorm.weight
bert.encoder.layer.10.attention.output.LayerNorm.bias
bert.encoder.layer.10.intermediate.dense.weight
bert.encoder.layer.10.intermediate.dense.bias
bert.encoder.layer.10.output.dense.weight
bert.encoder.layer.10.output.dense.bias
bert.encoder.layer.10.output.LayerNorm.weight
bert.encoder.layer.10.output.LayerNorm.bias
bert.encoder.layer.11.attention.self.query.weight
bert.encoder.layer.11.attention.self.query.bias
bert.encoder.layer.11.attention.self.key.weight
bert.encoder.layer.11.attention.self.key.bias
bert.encoder.layer.11.attention.self.value.weight
bert.encoder.layer.11.attention.self.value.bias
bert.encoder.layer.11.attention.output.dense.weight
bert.encoder.layer.11.attention.output.dense.bias
bert.encoder.layer.11.attention.output.LayerNorm.weight
bert.encoder.layer.11.attention.output.LayerNorm.bias
bert.encoder.layer.11.intermediate.dense.weight
bert.encoder.layer.11.intermediate.dense.bias
bert.encoder.layer.11.output.dense.weight
bert.encoder.layer.11.output.dense.bias
bert.encoder.layer.11.output.LayerNorm.weight
bert.encoder.layer.11.output.LayerNorm.bias
bert.pooler.dense.weight
bert.pooler.dense.bias
classifier.weight
classifier.bias
for name, param in model.named_parameters():

     # Trainable classification head
     if name.startswith("classifier"):
        param.requires_grad = True

      # Freeze everything else
     else:
        param.requires_grad = False
# We can check whether the model was correctly updated
for name, param in model.named_parameters():
     print(f"Parameter: {name} ----- {param.requires_grad}")
Parameter: bert.embeddings.word_embeddings.weight ----- False
Parameter: bert.embeddings.position_embeddings.weight ----- False
Parameter: bert.embeddings.token_type_embeddings.weight ----- False
Parameter: bert.embeddings.LayerNorm.weight ----- False
Parameter: bert.embeddings.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.0.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.0.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.0.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.0.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.0.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.0.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.0.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.0.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.0.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.0.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.0.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.0.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.0.output.dense.weight ----- False
Parameter: bert.encoder.layer.0.output.dense.bias ----- False
Parameter: bert.encoder.layer.0.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.0.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.1.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.1.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.1.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.1.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.1.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.1.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.1.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.1.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.1.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.1.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.1.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.1.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.1.output.dense.weight ----- False
Parameter: bert.encoder.layer.1.output.dense.bias ----- False
Parameter: bert.encoder.layer.1.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.1.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.2.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.2.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.2.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.2.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.2.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.2.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.2.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.2.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.2.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.2.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.2.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.2.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.2.output.dense.weight ----- False
Parameter: bert.encoder.layer.2.output.dense.bias ----- False
Parameter: bert.encoder.layer.2.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.2.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.3.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.3.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.3.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.3.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.3.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.3.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.3.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.3.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.3.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.3.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.3.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.3.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.3.output.dense.weight ----- False
Parameter: bert.encoder.layer.3.output.dense.bias ----- False
Parameter: bert.encoder.layer.3.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.3.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.4.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.4.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.4.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.4.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.4.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.4.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.4.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.4.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.4.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.4.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.4.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.4.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.4.output.dense.weight ----- False
Parameter: bert.encoder.layer.4.output.dense.bias ----- False
Parameter: bert.encoder.layer.4.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.4.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.5.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.5.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.5.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.5.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.5.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.5.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.5.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.5.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.5.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.5.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.5.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.5.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.5.output.dense.weight ----- False
Parameter: bert.encoder.layer.5.output.dense.bias ----- False
Parameter: bert.encoder.layer.5.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.5.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.6.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.6.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.6.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.6.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.6.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.6.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.6.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.6.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.6.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.6.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.6.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.6.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.6.output.dense.weight ----- False
Parameter: bert.encoder.layer.6.output.dense.bias ----- False
Parameter: bert.encoder.layer.6.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.6.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.7.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.7.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.7.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.7.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.7.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.7.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.7.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.7.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.7.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.7.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.7.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.7.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.7.output.dense.weight ----- False
Parameter: bert.encoder.layer.7.output.dense.bias ----- False
Parameter: bert.encoder.layer.7.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.7.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.8.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.8.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.8.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.8.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.8.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.8.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.8.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.8.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.8.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.8.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.8.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.8.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.8.output.dense.weight ----- False
Parameter: bert.encoder.layer.8.output.dense.bias ----- False
Parameter: bert.encoder.layer.8.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.8.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.9.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.9.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.9.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.9.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.9.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.9.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.9.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.9.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.9.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.9.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.9.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.9.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.9.output.dense.weight ----- False
Parameter: bert.encoder.layer.9.output.dense.bias ----- False
Parameter: bert.encoder.layer.9.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.9.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.10.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.10.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.10.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.10.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.10.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.10.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.10.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.10.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.10.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.10.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.10.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.10.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.10.output.dense.weight ----- False
Parameter: bert.encoder.layer.10.output.dense.bias ----- False
Parameter: bert.encoder.layer.10.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.10.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.11.attention.self.query.weight ----- False
Parameter: bert.encoder.layer.11.attention.self.query.bias ----- False
Parameter: bert.encoder.layer.11.attention.self.key.weight ----- False
Parameter: bert.encoder.layer.11.attention.self.key.bias ----- False
Parameter: bert.encoder.layer.11.attention.self.value.weight ----- False
Parameter: bert.encoder.layer.11.attention.self.value.bias ----- False
Parameter: bert.encoder.layer.11.attention.output.dense.weight ----- False
Parameter: bert.encoder.layer.11.attention.output.dense.bias ----- False
Parameter: bert.encoder.layer.11.attention.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.11.attention.output.LayerNorm.bias ----- False
Parameter: bert.encoder.layer.11.intermediate.dense.weight ----- False
Parameter: bert.encoder.layer.11.intermediate.dense.bias ----- False
Parameter: bert.encoder.layer.11.output.dense.weight ----- False
Parameter: bert.encoder.layer.11.output.dense.bias ----- False
Parameter: bert.encoder.layer.11.output.LayerNorm.weight ----- False
Parameter: bert.encoder.layer.11.output.LayerNorm.bias ----- False
Parameter: bert.pooler.dense.weight ----- False
Parameter: bert.pooler.dense.bias ----- False
Parameter: classifier.weight ----- True
Parameter: classifier.bias ----- True
from transformers import TrainingArguments, Trainer

# Trainer which executes the training process
trainer = Trainer(
   model=model,
   args=training_args,
   train_dataset=tokenized_train,
   eval_dataset=tokenized_test,
   tokenizer=tokenizer,
   data_collator=data_collator,
   compute_metrics=compute_metrics,
)
trainer.train()
[534/534 00:15, Epoch 1/1]
Step Training Loss
500 0.697000 

TrainOutput(global_step=534, training_loss=0.6962381677234664, metrics={'train_runtime': 15.234, 'train_samples_per_second': 559.931, 'train_steps_per_second': 35.053, 'total_flos': 227605451772240.0, 'train_loss': 0.6962381677234664, 'epoch': 1.0})
trainer.evaluate()
[67/67 00:01] 
{'eval_loss': 0.6823198795318604,
 'eval_f1': 0.637704918032787,
 'eval_runtime': 2.7203,
 'eval_samples_per_second': 391.865,
 'eval_steps_per_second': 24.629,
 'epoch': 1.0}

Freeze blocks 1-5

# We can check whether the model was correctly updated
for index, (name, param) in enumerate(model.named_parameters()):
     print(f"Parameter: {index}{name} ----- {param.requires_grad}")
Parameter: 0bert.embeddings.word_embeddings.weight ----- False
Parameter: 1bert.embeddings.position_embeddings.weight ----- False
Parameter: 2bert.embeddings.token_type_embeddings.weight ----- False
Parameter: 3bert.embeddings.LayerNorm.weight ----- False
Parameter: 4bert.embeddings.LayerNorm.bias ----- False
Parameter: 5bert.encoder.layer.0.attention.self.query.weight ----- False
Parameter: 6bert.encoder.layer.0.attention.self.query.bias ----- False
Parameter: 7bert.encoder.layer.0.attention.self.key.weight ----- False
Parameter: 8bert.encoder.layer.0.attention.self.key.bias ----- False
Parameter: 9bert.encoder.layer.0.attention.self.value.weight ----- False
Parameter: 10bert.encoder.layer.0.attention.self.value.bias ----- False
Parameter: 11bert.encoder.layer.0.attention.output.dense.weight ----- False
Parameter: 12bert.encoder.layer.0.attention.output.dense.bias ----- False
Parameter: 13bert.encoder.layer.0.attention.output.LayerNorm.weight ----- False
Parameter: 14bert.encoder.layer.0.attention.output.LayerNorm.bias ----- False
Parameter: 15bert.encoder.layer.0.intermediate.dense.weight ----- False
Parameter: 16bert.encoder.layer.0.intermediate.dense.bias ----- False
Parameter: 17bert.encoder.layer.0.output.dense.weight ----- False
Parameter: 18bert.encoder.layer.0.output.dense.bias ----- False
Parameter: 19bert.encoder.layer.0.output.LayerNorm.weight ----- False
Parameter: 20bert.encoder.layer.0.output.LayerNorm.bias ----- False
Parameter: 21bert.encoder.layer.1.attention.self.query.weight ----- False
Parameter: 22bert.encoder.layer.1.attention.self.query.bias ----- False
Parameter: 23bert.encoder.layer.1.attention.self.key.weight ----- False
Parameter: 24bert.encoder.layer.1.attention.self.key.bias ----- False
Parameter: 25bert.encoder.layer.1.attention.self.value.weight ----- False
Parameter: 26bert.encoder.layer.1.attention.self.value.bias ----- False
Parameter: 27bert.encoder.layer.1.attention.output.dense.weight ----- False
Parameter: 28bert.encoder.layer.1.attention.output.dense.bias ----- False
Parameter: 29bert.encoder.layer.1.attention.output.LayerNorm.weight ----- False
Parameter: 30bert.encoder.layer.1.attention.output.LayerNorm.bias ----- False
Parameter: 31bert.encoder.layer.1.intermediate.dense.weight ----- False
Parameter: 32bert.encoder.layer.1.intermediate.dense.bias ----- False
Parameter: 33bert.encoder.layer.1.output.dense.weight ----- False
Parameter: 34bert.encoder.layer.1.output.dense.bias ----- False
Parameter: 35bert.encoder.layer.1.output.LayerNorm.weight ----- False
Parameter: 36bert.encoder.layer.1.output.LayerNorm.bias ----- False
Parameter: 37bert.encoder.layer.2.attention.self.query.weight ----- False
Parameter: 38bert.encoder.layer.2.attention.self.query.bias ----- False
Parameter: 39bert.encoder.layer.2.attention.self.key.weight ----- False
Parameter: 40bert.encoder.layer.2.attention.self.key.bias ----- False
Parameter: 41bert.encoder.layer.2.attention.self.value.weight ----- False
Parameter: 42bert.encoder.layer.2.attention.self.value.bias ----- False
Parameter: 43bert.encoder.layer.2.attention.output.dense.weight ----- False
Parameter: 44bert.encoder.layer.2.attention.output.dense.bias ----- False
Parameter: 45bert.encoder.layer.2.attention.output.LayerNorm.weight ----- False
Parameter: 46bert.encoder.layer.2.attention.output.LayerNorm.bias ----- False
Parameter: 47bert.encoder.layer.2.intermediate.dense.weight ----- False
Parameter: 48bert.encoder.layer.2.intermediate.dense.bias ----- False
Parameter: 49bert.encoder.layer.2.output.dense.weight ----- False
Parameter: 50bert.encoder.layer.2.output.dense.bias ----- False
Parameter: 51bert.encoder.layer.2.output.LayerNorm.weight ----- False
Parameter: 52bert.encoder.layer.2.output.LayerNorm.bias ----- False
Parameter: 53bert.encoder.layer.3.attention.self.query.weight ----- False
Parameter: 54bert.encoder.layer.3.attention.self.query.bias ----- False
Parameter: 55bert.encoder.layer.3.attention.self.key.weight ----- False
Parameter: 56bert.encoder.layer.3.attention.self.key.bias ----- False
Parameter: 57bert.encoder.layer.3.attention.self.value.weight ----- False
Parameter: 58bert.encoder.layer.3.attention.self.value.bias ----- False
Parameter: 59bert.encoder.layer.3.attention.output.dense.weight ----- False
Parameter: 60bert.encoder.layer.3.attention.output.dense.bias ----- False
Parameter: 61bert.encoder.layer.3.attention.output.LayerNorm.weight ----- False
Parameter: 62bert.encoder.layer.3.attention.output.LayerNorm.bias ----- False
Parameter: 63bert.encoder.layer.3.intermediate.dense.weight ----- False
Parameter: 64bert.encoder.layer.3.intermediate.dense.bias ----- False
Parameter: 65bert.encoder.layer.3.output.dense.weight ----- False
Parameter: 66bert.encoder.layer.3.output.dense.bias ----- False
Parameter: 67bert.encoder.layer.3.output.LayerNorm.weight ----- False
Parameter: 68bert.encoder.layer.3.output.LayerNorm.bias ----- False
Parameter: 69bert.encoder.layer.4.attention.self.query.weight ----- False
Parameter: 70bert.encoder.layer.4.attention.self.query.bias ----- False
Parameter: 71bert.encoder.layer.4.attention.self.key.weight ----- False
Parameter: 72bert.encoder.layer.4.attention.self.key.bias ----- False
Parameter: 73bert.encoder.layer.4.attention.self.value.weight ----- False
Parameter: 74bert.encoder.layer.4.attention.self.value.bias ----- False
Parameter: 75bert.encoder.layer.4.attention.output.dense.weight ----- False
Parameter: 76bert.encoder.layer.4.attention.output.dense.bias ----- False
Parameter: 77bert.encoder.layer.4.attention.output.LayerNorm.weight ----- False
Parameter: 78bert.encoder.layer.4.attention.output.LayerNorm.bias ----- False
Parameter: 79bert.encoder.layer.4.intermediate.dense.weight ----- False
Parameter: 80bert.encoder.layer.4.intermediate.dense.bias ----- False
Parameter: 81bert.encoder.layer.4.output.dense.weight ----- False
Parameter: 82bert.encoder.layer.4.output.dense.bias ----- False
Parameter: 83bert.encoder.layer.4.output.LayerNorm.weight ----- False
Parameter: 84bert.encoder.layer.4.output.LayerNorm.bias ----- False
Parameter: 85bert.encoder.layer.5.attention.self.query.weight ----- False
Parameter: 86bert.encoder.layer.5.attention.self.query.bias ----- False
Parameter: 87bert.encoder.layer.5.attention.self.key.weight ----- False
Parameter: 88bert.encoder.layer.5.attention.self.key.bias ----- False
Parameter: 89bert.encoder.layer.5.attention.self.value.weight ----- False
Parameter: 90bert.encoder.layer.5.attention.self.value.bias ----- False
Parameter: 91bert.encoder.layer.5.attention.output.dense.weight ----- False
Parameter: 92bert.encoder.layer.5.attention.output.dense.bias ----- False
Parameter: 93bert.encoder.layer.5.attention.output.LayerNorm.weight ----- False
Parameter: 94bert.encoder.layer.5.attention.output.LayerNorm.bias ----- False
Parameter: 95bert.encoder.layer.5.intermediate.dense.weight ----- False
Parameter: 96bert.encoder.layer.5.intermediate.dense.bias ----- False
Parameter: 97bert.encoder.layer.5.output.dense.weight ----- False
Parameter: 98bert.encoder.layer.5.output.dense.bias ----- False
Parameter: 99bert.encoder.layer.5.output.LayerNorm.weight ----- False
Parameter: 100bert.encoder.layer.5.output.LayerNorm.bias ----- False
Parameter: 101bert.encoder.layer.6.attention.self.query.weight ----- False
Parameter: 102bert.encoder.layer.6.attention.self.query.bias ----- False
Parameter: 103bert.encoder.layer.6.attention.self.key.weight ----- False
Parameter: 104bert.encoder.layer.6.attention.self.key.bias ----- False
Parameter: 105bert.encoder.layer.6.attention.self.value.weight ----- False
Parameter: 106bert.encoder.layer.6.attention.self.value.bias ----- False
Parameter: 107bert.encoder.layer.6.attention.output.dense.weight ----- False
Parameter: 108bert.encoder.layer.6.attention.output.dense.bias ----- False
Parameter: 109bert.encoder.layer.6.attention.output.LayerNorm.weight ----- False
Parameter: 110bert.encoder.layer.6.attention.output.LayerNorm.bias ----- False
Parameter: 111bert.encoder.layer.6.intermediate.dense.weight ----- False
Parameter: 112bert.encoder.layer.6.intermediate.dense.bias ----- False
Parameter: 113bert.encoder.layer.6.output.dense.weight ----- False
Parameter: 114bert.encoder.layer.6.output.dense.bias ----- False
Parameter: 115bert.encoder.layer.6.output.LayerNorm.weight ----- False
Parameter: 116bert.encoder.layer.6.output.LayerNorm.bias ----- False
Parameter: 117bert.encoder.layer.7.attention.self.query.weight ----- False
Parameter: 118bert.encoder.layer.7.attention.self.query.bias ----- False
Parameter: 119bert.encoder.layer.7.attention.self.key.weight ----- False
Parameter: 120bert.encoder.layer.7.attention.self.key.bias ----- False
Parameter: 121bert.encoder.layer.7.attention.self.value.weight ----- False
Parameter: 122bert.encoder.layer.7.attention.self.value.bias ----- False
Parameter: 123bert.encoder.layer.7.attention.output.dense.weight ----- False
Parameter: 124bert.encoder.layer.7.attention.output.dense.bias ----- False
Parameter: 125bert.encoder.layer.7.attention.output.LayerNorm.weight ----- False
Parameter: 126bert.encoder.layer.7.attention.output.LayerNorm.bias ----- False
Parameter: 127bert.encoder.layer.7.intermediate.dense.weight ----- False
Parameter: 128bert.encoder.layer.7.intermediate.dense.bias ----- False
Parameter: 129bert.encoder.layer.7.output.dense.weight ----- False
Parameter: 130bert.encoder.layer.7.output.dense.bias ----- False
Parameter: 131bert.encoder.layer.7.output.LayerNorm.weight ----- False
Parameter: 132bert.encoder.layer.7.output.LayerNorm.bias ----- False
Parameter: 133bert.encoder.layer.8.attention.self.query.weight ----- False
Parameter: 134bert.encoder.layer.8.attention.self.query.bias ----- False
Parameter: 135bert.encoder.layer.8.attention.self.key.weight ----- False
Parameter: 136bert.encoder.layer.8.attention.self.key.bias ----- False
Parameter: 137bert.encoder.layer.8.attention.self.value.weight ----- False
Parameter: 138bert.encoder.layer.8.attention.self.value.bias ----- False
Parameter: 139bert.encoder.layer.8.attention.output.dense.weight ----- False
Parameter: 140bert.encoder.layer.8.attention.output.dense.bias ----- False
Parameter: 141bert.encoder.layer.8.attention.output.LayerNorm.weight ----- False
Parameter: 142bert.encoder.layer.8.attention.output.LayerNorm.bias ----- False
Parameter: 143bert.encoder.layer.8.intermediate.dense.weight ----- False
Parameter: 144bert.encoder.layer.8.intermediate.dense.bias ----- False
Parameter: 145bert.encoder.layer.8.output.dense.weight ----- False
Parameter: 146bert.encoder.layer.8.output.dense.bias ----- False
Parameter: 147bert.encoder.layer.8.output.LayerNorm.weight ----- False
Parameter: 148bert.encoder.layer.8.output.LayerNorm.bias ----- False
Parameter: 149bert.encoder.layer.9.attention.self.query.weight ----- False
Parameter: 150bert.encoder.layer.9.attention.self.query.bias ----- False
Parameter: 151bert.encoder.layer.9.attention.self.key.weight ----- False
Parameter: 152bert.encoder.layer.9.attention.self.key.bias ----- False
Parameter: 153bert.encoder.layer.9.attention.self.value.weight ----- False
Parameter: 154bert.encoder.layer.9.attention.self.value.bias ----- False
Parameter: 155bert.encoder.layer.9.attention.output.dense.weight ----- False
Parameter: 156bert.encoder.layer.9.attention.output.dense.bias ----- False
Parameter: 157bert.encoder.layer.9.attention.output.LayerNorm.weight ----- False
Parameter: 158bert.encoder.layer.9.attention.output.LayerNorm.bias ----- False
Parameter: 159bert.encoder.layer.9.intermediate.dense.weight ----- False
Parameter: 160bert.encoder.layer.9.intermediate.dense.bias ----- False
Parameter: 161bert.encoder.layer.9.output.dense.weight ----- False
Parameter: 162bert.encoder.layer.9.output.dense.bias ----- False
Parameter: 163bert.encoder.layer.9.output.LayerNorm.weight ----- False
Parameter: 164bert.encoder.layer.9.output.LayerNorm.bias ----- False
Parameter: 165bert.encoder.layer.10.attention.self.query.weight ----- False
Parameter: 166bert.encoder.layer.10.attention.self.query.bias ----- False
Parameter: 167bert.encoder.layer.10.attention.self.key.weight ----- False
Parameter: 168bert.encoder.layer.10.attention.self.key.bias ----- False
Parameter: 169bert.encoder.layer.10.attention.self.value.weight ----- False
Parameter: 170bert.encoder.layer.10.attention.self.value.bias ----- False
Parameter: 171bert.encoder.layer.10.attention.output.dense.weight ----- False
Parameter: 172bert.encoder.layer.10.attention.output.dense.bias ----- False
Parameter: 173bert.encoder.layer.10.attention.output.LayerNorm.weight ----- False
Parameter: 174bert.encoder.layer.10.attention.output.LayerNorm.bias ----- False
Parameter: 175bert.encoder.layer.10.intermediate.dense.weight ----- False
Parameter: 176bert.encoder.layer.10.intermediate.dense.bias ----- False
Parameter: 177bert.encoder.layer.10.output.dense.weight ----- False
Parameter: 178bert.encoder.layer.10.output.dense.bias ----- False
Parameter: 179bert.encoder.layer.10.output.LayerNorm.weight ----- False
Parameter: 180bert.encoder.layer.10.output.LayerNorm.bias ----- False
Parameter: 181bert.encoder.layer.11.attention.self.query.weight ----- False
Parameter: 182bert.encoder.layer.11.attention.self.query.bias ----- False
Parameter: 183bert.encoder.layer.11.attention.self.key.weight ----- False
Parameter: 184bert.encoder.layer.11.attention.self.key.bias ----- False
Parameter: 185bert.encoder.layer.11.attention.self.value.weight ----- False
Parameter: 186bert.encoder.layer.11.attention.self.value.bias ----- False
Parameter: 187bert.encoder.layer.11.attention.output.dense.weight ----- False
Parameter: 188bert.encoder.layer.11.attention.output.dense.bias ----- False
Parameter: 189bert.encoder.layer.11.attention.output.LayerNorm.weight ----- False
Parameter: 190bert.encoder.layer.11.attention.output.LayerNorm.bias ----- False
Parameter: 191bert.encoder.layer.11.intermediate.dense.weight ----- False
Parameter: 192bert.encoder.layer.11.intermediate.dense.bias ----- False
Parameter: 193bert.encoder.layer.11.output.dense.weight ----- False
Parameter: 194bert.encoder.layer.11.output.dense.bias ----- False
Parameter: 195bert.encoder.layer.11.output.LayerNorm.weight ----- False
Parameter: 196bert.encoder.layer.11.output.LayerNorm.bias ----- False
Parameter: 197bert.pooler.dense.weight ----- False
Parameter: 198bert.pooler.dense.bias ----- False
Parameter: 199classifier.weight ----- True
Parameter: 200classifier.bias ----- True
# Load model
model_id = "bert-base-cased"
model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=2)
tokenizer = AutoTokenizer.from_pretrained(model_id)

# Encoder block 10 starts at index 165 and
# we freeze everything before that block
for index, (name, param) in enumerate(model.named_parameters()):
    if index < 165:
        param.requires_grad = False

# Trainer which executes the training process
trainer = Trainer(
   model=model,
   args=training_args,
   train_dataset=tokenized_train,
   eval_dataset=tokenized_test,
   tokenizer=tokenizer,
   data_collator=data_collator,
   compute_metrics=compute_metrics,
)
trainer.train()
trainer.evaluate()
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
[534/534 00:21, Epoch 1/1]
Step Training Loss
500 0.474600

[67/67 00:01] 
{'eval_loss': 0.4092540740966797,
 'eval_f1': 0.8141086749285034,
 'eval_runtime': 2.7437,
 'eval_samples_per_second': 388.523,
 'eval_steps_per_second': 24.419,
 'epoch': 1.0}

[BONUS] Freeze blocks

# scores = []
# for index in range(12):
#     # Re-load model
#     model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=2)
#     tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

#     # Freeze encoder blocks 0-index
#     for name, param in model.named_parameters():
#         if "layer" in name:
#             layer_nr = int(name.split("layer")[1].split(".")[1])
#             if layer_nr <= index:
#                 param.requires_grad = False
#         else:
#             param.requires_grad = True

#     # Train
#     trainer = Trainer(
#       model=model,
#       args=training_args,
#       train_dataset=tokenized_train,
#       eval_dataset=tokenized_test,
#       tokenizer=tokenizer,
#       data_collator=data_collator,
#       compute_metrics=compute_metrics,
#     )
#     trainer.train()

#     # Evaluate
#     score = trainer.evaluate()["eval_f1"]
#     scores.append(score)
# scores
[0.8541862652869239,
 0.8525519848771267,
 0.8514664143803217,
 0.8506616257088847,
 0.8398104265402844,
 0.8391345249294448,
 0.8377358490566037,
 0.8433962264150944,
 0.8258801141769743,
 0.816247582205029,
 0.7917485265225934,
 0.7019400352733686]
# import matplotlib.pyplot as plt
# import numpy as np

# # Create Figure
# plt.figure(figsize=(8,4))

# # Prepare Data
# x = [f"0-{index}" for index in range(12)]
# x[0] = "None"
# x[-1] = "All"
# y = [
#     0.8541862652869239,
#     0.8525519848771267,
#     0.8514664143803217,
#     0.8506616257088847,
#     0.8398104265402844,
#     0.8391345249294448,
#     0.8377358490566037,
#     0.8433962264150944,
#     0.8258801141769743,
#     0.816247582205029,
#     0.7917485265225934,
#     0.7019400352733686
# ][::-1]

# # Stylize Figure
# plt.grid(color='#ECEFF1')
# plt.axvline(x=4, color="#EC407A", linestyle="--")
# plt.title("Effect of Frozen Encoder Blocks on Training Performance")
# plt.ylabel("F1-score")
# plt.xlabel("Trainable encoder blocks")

# # Plot Data
# plt.plot(x, y, color="black")

# # Additional Annotation
# plt.annotate(
#     'Performance stabilizing',
#     xy=(4, y[4]),
#     xytext=(4.5, y[4]-.05),
#     arrowprops=dict(
#         arrowstyle="-|>",
#         connectionstyle="arc3",
#         color="#00ACC1")
# )
# plt.savefig("multiple_frozen_blocks.png", dpi=300, bbox_inches='tight')

Few-shot Classification

from setfit import sample_dataset

# We simulate a few-shot setting by sampling 16 examples per class
sampled_train_data = sample_dataset(tomatoes["train"], num_samples=16)
from setfit import SetFitModel

# Load a pre-trained SentenceTransformer model
model = SetFitModel.from_pretrained("sentence-transformers/all-mpnet-base-v2")
/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)
/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(
model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.
from setfit import TrainingArguments as SetFitTrainingArguments
from setfit import Trainer as SetFitTrainer

# Define training arguments
args = SetFitTrainingArguments(
    num_epochs=3, # The number of epochs to use for contrastive learning
    num_iterations=20  # The number of text pairs to generate
)
args.eval_strategy = args.evaluation_strategy

# Create trainer
trainer = SetFitTrainer(
    model=model,
    args=args,
    train_dataset=sampled_train_data,
    eval_dataset=test_data,
    metric="f1"
)
# from setfit import SetFitTrainer

# # Create trainer
# trainer = SetFitTrainer(
#     model=model,
#     train_dataset=sampled_train_data,
#     eval_dataset=test_data,
#     metric="f1",
#     num_epochs=3, # The number of epochs to use for contrastive learning
# )
# Training loop
trainer.train()
***** Running training *****
  Num unique pairs = 1280
  Batch size = 16
  Num epochs = 3
  Total optimization steps = 240
[240/240 00:37, Epoch 3/0]
Step Training Loss 

# Evaluate the model on our test data
trainer.evaluate()
***** Running evaluation *****
{'f1': 0.8363988383349468}
model.model_head
/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)
LogisticRegression()
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

MLM

from transformers import AutoTokenizer, AutoModelForMaskedLM

# Load model for Masked Language Modeling (MLM)
model = AutoModelForMaskedLM.from_pretrained("bert-base-cased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']
- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
def preprocess_function(examples):
   return tokenizer(examples["text"], truncation=True)

# Tokenize data
tokenized_train = train_data.map(preprocess_function, batched=True)
tokenized_train = tokenized_train.remove_columns("label")
tokenized_test = test_data.map(preprocess_function, batched=True)
tokenized_test = tokenized_test.remove_columns("label")
from transformers import DataCollatorForLanguageModeling

# Masking Tokens
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=True,
    mlm_probability=0.15
)
# from transformers import DataCollatorForWholeWordMask

# # Masking Whole Words
# data_collator = DataCollatorForWholeWordMask(
#     tokenizer=tokenizer,
#     mlm=True,
#     mlm_probability=0.15
# )
# Training arguments for parameter tuning
training_args = TrainingArguments(
   "model",
   learning_rate=2e-5,
   per_device_train_batch_size=16,
   per_device_eval_batch_size=16,
   num_train_epochs=10,
   weight_decay=0.01,
   save_strategy="epoch",
   report_to="none"
)


# Initialize Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train,
    eval_dataset=tokenized_test,
    tokenizer=tokenizer,
    data_collator=data_collator
)
# Save pre-trained tokenizer
tokenizer.save_pretrained("mlm")

# Train model
trainer.train()

# Save updated model
model.save_pretrained("mlm")
[5340/5340 12:10, Epoch 10/10]
Step Training Loss
500 2.601700
1000 2.377500
1500 2.313100
2000 2.187500
2500 2.150400
3000 2.096100
3500 2.059500
4000 1.990300
4500 1.986100
5000 1.958500 

from transformers import pipeline

# Load and create predictions
mask_filler = pipeline("fill-mask", model="bert-base-cased")
preds = mask_filler("What a horrible [MASK]!")

# Print results
for pred in preds:
    print(f">>> {pred['sequence']}")
/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
  and should_run_async(code)
Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']
- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
>>> What a horrible idea!
>>> What a horrible dream!
>>> What a horrible thing!
>>> What a horrible day!
>>> What a horrible thought!
# Load and create predictions
mask_filler = pipeline("fill-mask", model="mlm")
preds = mask_filler("What a horrible [MASK]!")

# Print results
for pred in preds:
    print(f">>> {pred['sequence']}")
>>> What a horrible movie!
>>> What a horrible film!
>>> What a horrible mess!
>>> What a horrible comedy!
>>> What a horrible story!

Named Entity Recognition

Here are a number of interesting datasets you can also explore for NER: * tner/mit_movie_trivia * tner/mit_restaurant * wnut_17 * conll2003

from transformers import AutoModelForTokenClassification, AutoTokenizer
from transformers import DataCollatorWithPadding
from transformers import TrainingArguments, Trainer
import numpy as np
# The CoNLL-2003 dataset for NER
dataset = load_dataset("conll2003", trust_remote_code=True)
The repository for conll2003 contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/conll2003.
You can avoid this prompt in future by passing the argument `trust_remote_code=True`.

Do you wish to run the custom code? [y/N] y
example = dataset["train"][848]
example
{'id': '848',
 'tokens': ['Dean',
  'Palmer',
  'hit',
  'his',
  '30th',
  'homer',
  'for',
  'the',
  'Rangers',
  '.'],
 'pos_tags': [22, 22, 38, 29, 16, 21, 15, 12, 23, 7],
 'chunk_tags': [11, 12, 21, 11, 12, 12, 13, 11, 12, 0],
 'ner_tags': [1, 2, 0, 0, 0, 0, 0, 0, 3, 0]}
label2id = {
    'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4,
    'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8
}
id2label = {index: label for label, index in label2id.items()}
label2id
{'O': 0,
 'B-PER': 1,
 'I-PER': 2,
 'B-ORG': 3,
 'I-ORG': 4,
 'B-LOC': 5,
 'I-LOC': 6,
 'B-MISC': 7,
 'I-MISC': 8}
from transformers import AutoModelForTokenClassification

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

# Load model
model = AutoModelForTokenClassification.from_pretrained(
    "bert-base-cased",
    num_labels=len(id2label),
    id2label=id2label,
    label2id=label2id
)
Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
# Split individual tokens into sub-tokens
token_ids = tokenizer(example["tokens"], is_split_into_words=True)["input_ids"]
sub_tokens = tokenizer.convert_ids_to_tokens(token_ids)
sub_tokens
['[CLS]',
 'Dean',
 'Palmer',
 'hit',
 'his',
 '30th',
 'home',
 '##r',
 'for',
 'the',
 'Rangers',
 '.',
 '[SEP]']
def align_labels(examples):
    token_ids = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
    labels = examples["ner_tags"]

    updated_labels = []
    for index, label in enumerate(labels):

        # Map tokens to their respective word
        word_ids = token_ids.word_ids(batch_index=index)
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:

            # The start of a new word
            if word_idx != previous_word_idx:

                previous_word_idx = word_idx
                updated_label = -100 if word_idx is None else label[word_idx]
                label_ids.append(updated_label)

            # Special token is -100
            elif word_idx is None:
                label_ids.append(-100)

            # If the label is B-XXX we change it to I-XXX
            else:
                updated_label = label[word_idx]
                if updated_label % 2 == 1:
                    updated_label += 1
                label_ids.append(updated_label)

        updated_labels.append(label_ids)

    token_ids["labels"] = updated_labels
    return token_ids

tokenized = dataset.map(align_labels, batched=True)
# Difference between original and updated labels
print(f"Original: {example['ner_tags']}")
print(f"Updated: {tokenized['train'][848]['labels']}")
Original: [1, 2, 0, 0, 0, 0, 0, 0, 3, 0]
Updated: [-100, 1, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, -100]
import evaluate

# Load sequential evaluation
seqeval = evaluate.load("seqeval")

def compute_metrics(eval_pred):
    # Create predictions
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=2)

    true_predictions = []
    true_labels = []

    # Document-level iteration
    for prediction, label in zip(predictions, labels):

      # token-level iteration
      for token_prediction, token_label in zip(prediction, label):

        # We ignore special tokens
        if token_label != -100:
          true_predictions.append([id2label[token_prediction]])
          true_labels.append([id2label[token_label]])

    results = seqeval.compute(predictions=true_predictions, references=true_labels)
    return {"f1": results["overall_f1"]}
from transformers import DataCollatorForTokenClassification

# Token-classification Data Collator
data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)
# Training arguments for parameter tuning
training_args = TrainingArguments(
   "model",
   learning_rate=2e-5,
   per_device_train_batch_size=16,
   per_device_eval_batch_size=16,
   num_train_epochs=1,
   weight_decay=0.01,
   save_strategy="epoch",
   report_to="none"
)

# Initialize Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized["train"],
    eval_dataset=tokenized["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)
trainer.train()
[878/878 02:49, Epoch 1/1]
Step Training Loss
500 0.047500 

TrainOutput(global_step=878, training_loss=0.04094860494001037, metrics={'train_runtime': 169.4752, 'train_samples_per_second': 82.85, 'train_steps_per_second': 5.181, 'total_flos': 351240792638148.0, 'train_loss': 0.04094860494001037, 'epoch': 1.0})
# Evaluate the model on our test data
trainer.evaluate()
[216/216 00:09] 
{'eval_loss': 0.16888542473316193,
 'eval_f1': 0.9180087380808113,
 'eval_runtime': 14.5731,
 'eval_samples_per_second': 236.943,
 'eval_steps_per_second': 14.822,
 'epoch': 1.0}
from transformers import pipeline

# Save our fine-tuned model
trainer.save_model("ner_model")

# Run inference on the fine-tuned model
token_classifier = pipeline(
    "token-classification",
    model="ner_model",
)
token_classifier("My name is Maarten.")
[{'entity': 'B-PER',
  'score': 0.99534035,
  'index': 4,
  'word': 'Ma',
  'start': 11,
  'end': 13},
 {'entity': 'I-PER',
  'score': 0.9928328,
  'index': 5,
  'word': '##arte',
  'start': 13,
  'end': 17},
 {'entity': 'I-PER',
  'score': 0.9954301,
  'index': 6,
  'word': '##n',
  'start': 17,
  'end': 18}]
Back to top